require(class)

df_train = readRDS("train.rds")
df_test = readRDS("test.rds")

sort(sapply(df_train, function(x) length(unique(x))), decr=T)[1:10]
sort(sapply(df_test, function(x) length(unique(x))), decr=T)[1:10]

    id   f526   f715   f523   f449   f607   f610   f717   f217     f3 
105471 105446 105436 105418 105407 105261 105261 105261 105213 105209 

length(intersect(df_train$f526, df_test$f526))



length(unique(df_test$f275))

a=tapply(df_train$loss>0,df_train$f275, functienon(x) c(length(x), mean(x))) - 

for(i in 1:length(a)) if(a[[i]][1]>50) print(a[[i]][2])


edit(df_train)

#what is f521, some kind of sequence number? does f520 represent an outstanding balance? f521 some kind of pct paid?
#is f532 some kind of sequence number?

#conjecture - 275 is loan id, 521 is a sequence number

#df2 = df_train[df_train$f275!=0,]

#length(unique(paste(df2$f275, df2$f521,sep="-")))

ord = order(df_train$f275, df_train$f521)
id = df_train$f275[ord]
loss_ind=as.integer(df_train$loss>0)[ord]
next_id = rep(NA, length(id))
next_id[length(id)] = 0
for(i in 1:length(id)-1) next_id[i] = id[i+1]
u=data.frame(id, next_id,seq_num=df_train$f521[ord], flag=(id!=next_id),loss_ind)

tapply(u$loss_ind, u$flag, mean)

df2 = data.frame(df_train[ord,], flag=u$flag, loss_ind=u$loss_ind)

df3=df2[df2$flag==T,]

df4 = data.frame(lapply(df3, mean_impute))
1.237565e+21
sort(sapply(df4[,2:780],function(x) mean(df4$loss[x >= quantile(x, .99)]>0)), decr=T)[2:15]

pred = ifelse(df2$flag, ifelse(df2$f277>2.860455e+22, 1.97,0),0)

mean(df_train$loss)
mean(na.omit(abs(df_train$loss-pred)))



